<!doctype html>



  


<html class="theme-next pisces use-motion">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />












  
  
  <link href="/vendors/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  

  

  

  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/vendors/font-awesome/css/font-awesome.min.css?v=4.4.0" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.0.1" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="Java,Kafka," />





  <link rel="alternate" href="/atom.xml" title="crossoverJie's Blog" type="application/atom+xml" />




  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.0.1" />






<meta name="description" content="前言写这篇文章的起因是由于之前的一篇关于Kafka异常消费，当时为了解决问题不得不使用临时的方案。
总结起来归根结底还是对Kafka不熟悉导致的，加上平时工作的需要，之后就花些时间看了Kafka相关的资料。
何时使用MQ谈到Kafka就不得不提到MQ，是属于消息队列的一种。作为一种基础中间件在互联网项目中有着大量的使用。
一种技术的产生自然是为了解决某种需求，通常来说是以下场景：


需要跨进程">
<meta property="og:type" content="article">
<meta property="og:title" content="SSM(十七) MQ应用">
<meta property="og:url" content="http://crossoverjie.top/2017/10/20/SSM17/index.html">
<meta property="og:site_name" content="crossoverJie's Blog">
<meta property="og:description" content="前言写这篇文章的起因是由于之前的一篇关于Kafka异常消费，当时为了解决问题不得不使用临时的方案。
总结起来归根结底还是对Kafka不熟悉导致的，加上平时工作的需要，之后就花些时间看了Kafka相关的资料。
何时使用MQ谈到Kafka就不得不提到MQ，是属于消息队列的一种。作为一种基础中间件在互联网项目中有着大量的使用。
一种技术的产生自然是为了解决某种需求，通常来说是以下场景：


需要跨进程">
<meta property="og:image" content="https://i.loli.net/2019/05/08/5cd1ba77119c6.jpg">
<meta property="og:image" content="https://i.loli.net/2019/05/08/5cd1ba7910bc2.jpg">
<meta property="og:image" content="https://i.loli.net/2019/05/08/5cd1ba7c8876f.jpg">
<meta property="og:image" content="https://i.loli.net/2019/05/08/5cd1ba8705aac.jpg">
<meta property="og:image" content="https://i.loli.net/2019/05/08/5cd1ba8b41500.jpg">
<meta property="og:updated_time" content="2019-05-07T17:05:00.067Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="SSM(十七) MQ应用">
<meta name="twitter:description" content="前言写这篇文章的起因是由于之前的一篇关于Kafka异常消费，当时为了解决问题不得不使用临时的方案。
总结起来归根结底还是对Kafka不熟悉导致的，加上平时工作的需要，之后就花些时间看了Kafka相关的资料。
何时使用MQ谈到Kafka就不得不提到MQ，是属于消息队列的一种。作为一种基础中间件在互联网项目中有着大量的使用。
一种技术的产生自然是为了解决某种需求，通常来说是以下场景：


需要跨进程">
<meta name="twitter:image" content="https://i.loli.net/2019/05/08/5cd1ba77119c6.jpg">



<script type="text/javascript" id="hexo.configuration">
  var NexT = window.NexT || {};
  var CONFIG = {
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: 555390,
      author: 'crossoverJie'
    }
  };
</script>




  <link rel="canonical" href="http://crossoverjie.top/2017/10/20/SSM17/"/>

  <title> SSM(十七) MQ应用 | crossoverJie's Blog </title>
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="en">

  








  <div style="display: none;">
    <script src="https://s6.cnzz.com/stat.php?id=1259025147&web_id=1259025147" type="text/javascript"></script>
  </div>





  
  
    
  

  <div class="container one-collumn sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">crossoverJie's Blog</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle">baller</p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            Home
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            Categories
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            About
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            Archives
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            Tags
          </a>
        </li>
      
        
        <li class="menu-item menu-item-photo">
          <a href="/favourite" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-photo"></i> <br />
            
            photo
          </a>
        </li>
      
        
        <li class="menu-item menu-item-jcsprout">
          <a href="https://crossoverjie.top/JCSprout/#/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-leaf"></i> <br />
            
            JCSprout
          </a>
        </li>
      
        
        <li class="menu-item menu-item-cicada">
          <a href="https://github.com/TogetherOS/cicada" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-github"></i> <br />
            
            cicada
          </a>
        </li>
      
        
        <li class="menu-item menu-item-cim">
          <a href="https://github.com/crossoverjie/cim" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-far fa-comment"></i> <br />
            
            CIM
          </a>
        </li>
      
        
        <li class="menu-item menu-item-vlog">
          <a href="https://space.bilibili.com/42339430" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-youtube"></i> <br />
            
            Vlog
          </a>
        </li>
      

      
        <li class="menu-item menu-item-search">
          
            <a href="#" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />
            
            Search
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>


    </div>
  
</nav>

 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                SSM(十七) MQ应用
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">Posted on</span>
            <time itemprop="dateCreated" datetime="2017-10-20T01:01:54+08:00" content="2017-10-20">
              2017-10-20
            </time>
          </span>

          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/SSM/" itemprop="url" rel="index">
                    <span itemprop="name">SSM</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
              <span class="post-comments-count">
                &nbsp; | &nbsp;
                <a href="/2017/10/20/SSM17/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2017/10/20/SSM17/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          

          
          
             <span id="/2017/10/20/SSM17/" class="leancloud_visitors" data-flag-title="SSM(十七) MQ应用">
               &nbsp; | &nbsp;
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               <span class="post-meta-item-text">visitors </span>
               <span class="leancloud-visitors-count"></span>
              </span>
          

          
              &nbsp; | &nbsp;
              <span class="page-pv">本文总阅读量
              <span class="busuanzi-value" id="busuanzi_value_page_pv" ></span>次
              </span>
          
        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        <p><img src="https://i.loli.net/2019/05/08/5cd1ba77119c6.jpg" alt=""></p>
<h1 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h1><p>写这篇文章的起因是由于之前的一篇关于<code>Kafka</code><a href="http://crossoverjie.top/2017/09/05/SSM16/">异常消费</a>，当时为了解决问题不得不使用临时的方案。</p>
<p>总结起来归根结底还是对Kafka不熟悉导致的，加上平时工作的需要，之后就花些时间看了<code>Kafka</code>相关的资料。</p>
<h1 id="何时使用MQ"><a href="#何时使用MQ" class="headerlink" title="何时使用MQ"></a>何时使用MQ</h1><p>谈到<code>Kafka</code>就不得不提到MQ，是属于消息队列的一种。作为一种基础中间件在互联网项目中有着大量的使用。</p>
<p>一种技术的产生自然是为了解决某种需求，通常来说是以下场景：</p>
<blockquote>
<ul>
<li>需要跨进程通信：B系统需要A系统的输出作为输入参数。</li>
<li>当A系统的输出能力远远大于B系统的处理能力。</li>
</ul>
</blockquote>
<p>针对于第一种情况有两种方案:</p>
<ul>
<li>使用<code>RPC</code>远程调用,A直接调用B。</li>
<li>使用<code>MQ</code>,A发布消息到<code>MQ</code>,B订阅该消息。</li>
</ul>
<p>当我们的需求是:A调用B实时响应，并且实时关心响应结果则使用<code>RPC</code>，这种情况就得使用同步调用。</p>
<a id="more"></a>
<p>反之当我们并不关心调用之后的执行结果，并且有可能被调用方的执行非常耗时，这种情况就非常适合用<code>MQ</code>来达到异步调用目的。</p>
<p>比如常见的登录场景就只能用同步调用的方式，因为这个过程需要实时的响应结果，总不能在用户点了登录之后排除网络原因之外再额外的等几秒吧。</p>
<p>但类似于用户登录需要奖励积分的情况则使用<code>MQ</code>会更好，因为登录并不关系积分的情况，只需要发个消息到<code>MQ</code>,处理积分的服务订阅处理即可，这样还可以解决积分系统故障带来的雪崩效应。</p>
<p><code>MQ</code>还有一个基础功能则是<strong>限流削峰</strong>，这对于大流量的场景如果将请求直接调用到B系统则非常有可能使B系统出现不可用的情况。这种场景就非常适合将请求放入<code>MQ</code>，不但可以利用<code>MQ</code>削峰还尽可能的保证系统的高可用。</p>
<h1 id="Kafka简介"><a href="#Kafka简介" class="headerlink" title="Kafka简介"></a>Kafka简介</h1><p>本次重点讨论下<code>Kafka</code>。<br>简单来说<code>Kafka</code>是一个支持水平扩展，高吞吐率的分布式消息系统。</p>
<p><code>Kafka</code>的常用知识:</p>
<ul>
<li><p><code>Topic</code>:生产者和消费者的交互都是围绕着一个<code>Topic</code>进行的，通常来说是由业务来进行区分，由生产消费者协商之后进行创建。</p>
</li>
<li><p><code>Partition</code>(分区):是<code>Topic</code>下的组成，通常一个<code>Topic</code>下有一个或多个分区，消息生产之后会按照一定的算法负载到每个分区，所以分区也是<code>Kafka</code>性能的关键。当发现性能不高时便可考虑新增分区。</p>
</li>
</ul>
<p>结构图如下:<br><img src="https://i.loli.net/2019/05/08/5cd1ba7910bc2.jpg" alt=""></p>
<h1 id="创建Topic"><a href="#创建Topic" class="headerlink" title="创建Topic"></a>创建<code>Topic</code></h1><p><code>Kafka</code>的安装官网有非常详细的讲解。这里谈一下在日常开发中常见的一些操作，比如创建<code>Topic</code>：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sh bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic `test`</div></pre></td></tr></table></figure>
<p>创建了三个分区的<code>test</code>主题。</p>
<p>使用</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sh bin/kafka-topics.sh --list --zookeeper localhost:2181</div></pre></td></tr></table></figure>
<p>可以列出所有的<code>Topic</code>。</p>
<h1 id="Kafka生产者"><a href="#Kafka生产者" class="headerlink" title="Kafka生产者"></a>Kafka生产者</h1><p>使用<code>kafka</code>官方所提供的<code>Java API</code>来进行消息生产，实际使用中编码实现更为常用:</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div></pre></td><td class="code"><pre><div class="line"><span class="comment">/** Kafka生产者</span></div><div class="line"> * <span class="doctag">@author</span> crossoverJie</div><div class="line"> */</div><div class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">Producer</span> </span>&#123;</div><div class="line">    <span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">final</span> Logger LOGGER = LoggerFactory.getLogger(Producer.class);</div><div class="line"></div><div class="line">    <span class="comment">/**</span></div><div class="line">     * 消费配置文件</div><div class="line">     */</div><div class="line">    <span class="keyword">private</span> <span class="keyword">static</span> String consumerProPath;</div><div class="line"></div><div class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> <span class="keyword">throws</span> IOException </span>&#123;</div><div class="line">        <span class="comment">// set up the producer</span></div><div class="line">        consumerProPath = System.getProperty(<span class="string">"product_path"</span>);</div><div class="line">        KafkaProducer&lt;String, String&gt; producer = <span class="keyword">null</span>;</div><div class="line">        <span class="keyword">try</span> &#123;</div><div class="line">            FileInputStream inputStream = <span class="keyword">new</span> FileInputStream(<span class="keyword">new</span> File(consumerProPath));</div><div class="line">            Properties properties = <span class="keyword">new</span> Properties();</div><div class="line">            properties.load(inputStream);</div><div class="line">            producer = <span class="keyword">new</span> KafkaProducer&lt;String, String&gt;(properties);</div><div class="line"></div><div class="line">        &#125; <span class="keyword">catch</span> (IOException e) &#123;</div><div class="line">            LOGGER.error(<span class="string">"load config error"</span>, e);</div><div class="line">        &#125;</div><div class="line"></div><div class="line">        <span class="keyword">try</span> &#123;</div><div class="line">            <span class="comment">// send lots of messages</span></div><div class="line">            <span class="keyword">for</span> (<span class="keyword">int</span> i=<span class="number">0</span> ;i&lt;<span class="number">100</span> ; i++)&#123;</div><div class="line">                producer.send(<span class="keyword">new</span> ProducerRecord&lt;String, String&gt;(</div><div class="line">                        <span class="string">"topic_optimization"</span>, i+<span class="string">""</span>, i+<span class="string">""</span>));</div><div class="line"></div><div class="line">            &#125;</div><div class="line">        &#125; <span class="keyword">catch</span> (Throwable throwable) &#123;</div><div class="line">            System.out.printf(<span class="string">"%s"</span>, throwable.getStackTrace());</div><div class="line">        &#125; <span class="keyword">finally</span> &#123;</div><div class="line">            producer.close();</div><div class="line">        &#125;</div><div class="line"></div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<p>再配合以下启动参数即可发送消息:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">-Dproduct_path=/xxx/producer.properties</div></pre></td></tr></table></figure></p>
<p>以及生产者的配置文件:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div></pre></td><td class="code"><pre><div class="line">#集群地址，可以多个</div><div class="line">bootstrap.servers=10.19.13.51:9094</div><div class="line">acks=all</div><div class="line">retries=0</div><div class="line">batch.size=16384</div><div class="line">auto.commit.interval.ms=1000</div><div class="line">linger.ms=0</div><div class="line">key.serializer=org.apache.kafka.common.serialization.StringSerializer</div><div class="line">value.serializer=org.apache.kafka.common.serialization.StringSerializer</div><div class="line">block.on.buffer.full=true</div></pre></td></tr></table></figure></p>
<p>具体的配置说明详见此处:<a href="https://kafka.apache.org/0100/documentation.html#theproducer" target="_blank" rel="external">https://kafka.apache.org/0100/documentation.html#theproducer</a></p>
<p>流程非常简单，其实就是一些<code>API</code>的调用。</p>
<p>消息发完之后可以通过以下命令查看队列内的情况:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sh kafka-consumer-groups.sh --bootstrap-server localhost:9094 --describe --group group1</div></pre></td></tr></table></figure></p>
<p><img src="https://i.loli.net/2019/05/08/5cd1ba7c8876f.jpg" alt=""><br>其中的<code>lag</code>便是队列里的消息数量。</p>
<h1 id="Kafka消费者"><a href="#Kafka消费者" class="headerlink" title="Kafka消费者"></a>Kafka消费者</h1><p>有了生产者自然也少不了消费者，这里首先针对单线程消费:</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div><div class="line">45</div><div class="line">46</div><div class="line">47</div><div class="line">48</div><div class="line">49</div><div class="line">50</div><div class="line">51</div><div class="line">52</div><div class="line">53</div><div class="line">54</div><div class="line">55</div><div class="line">56</div><div class="line">57</div><div class="line">58</div><div class="line">59</div><div class="line">60</div><div class="line">61</div><div class="line">62</div><div class="line">63</div><div class="line">64</div><div class="line">65</div><div class="line">66</div><div class="line">67</div><div class="line">68</div><div class="line">69</div><div class="line">70</div><div class="line">71</div><div class="line">72</div><div class="line">73</div><div class="line">74</div><div class="line">75</div><div class="line">76</div><div class="line">77</div><div class="line">78</div><div class="line">79</div><div class="line">80</div><div class="line">81</div><div class="line">82</div><div class="line">83</div><div class="line">84</div><div class="line">85</div><div class="line">86</div><div class="line">87</div><div class="line">88</div><div class="line">89</div><div class="line">90</div><div class="line">91</div><div class="line">92</div><div class="line">93</div><div class="line">94</div><div class="line">95</div><div class="line">96</div><div class="line">97</div><div class="line">98</div><div class="line">99</div><div class="line">100</div><div class="line">101</div><div class="line">102</div><div class="line">103</div><div class="line">104</div><div class="line">105</div><div class="line">106</div></pre></td><td class="code"><pre><div class="line"><span class="comment">/**</span></div><div class="line"> * Function:kafka官方消费</div><div class="line"> *</div><div class="line"> * <span class="doctag">@author</span> crossoverJie</div><div class="line"> *         Date: 2017/10/19 01:11</div><div class="line"> * <span class="doctag">@since</span> JDK 1.8</div><div class="line"> */</div><div class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">KafkaOfficialConsumer</span> </span>&#123;</div><div class="line">    <span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">final</span> Logger LOGGER = LoggerFactory.getLogger(KafkaOfficialConsumer.class);</div><div class="line"></div><div class="line">    <span class="comment">/**</span></div><div class="line">     * 日志文件地址</div><div class="line">     */</div><div class="line">    <span class="keyword">private</span> <span class="keyword">static</span> String logPath;</div><div class="line"></div><div class="line">    <span class="comment">/**</span></div><div class="line">     * 主题名称</div><div class="line">     */</div><div class="line">    <span class="keyword">private</span> <span class="keyword">static</span> String topic;</div><div class="line"></div><div class="line">    <span class="comment">/**</span></div><div class="line">     * 消费配置文件</div><div class="line">     */</div><div class="line">    <span class="keyword">private</span> <span class="keyword">static</span> String consumerProPath ;</div><div class="line"></div><div class="line"></div><div class="line">    <span class="comment">/**</span></div><div class="line">     * 初始化参数校验</div><div class="line">     * <span class="doctag">@return</span></div><div class="line">     */</div><div class="line">    <span class="function"><span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">boolean</span> <span class="title">initCheck</span><span class="params">()</span> </span>&#123;</div><div class="line">        topic = System.getProperty(<span class="string">"topic"</span>) ;</div><div class="line">        logPath = System.getProperty(<span class="string">"log_path"</span>) ;</div><div class="line">        consumerProPath = System.getProperty(<span class="string">"consumer_pro_path"</span>) ;</div><div class="line">        <span class="keyword">if</span> (StringUtil.isEmpty(topic) || logPath.isEmpty()) &#123;</div><div class="line">            LOGGER.error(<span class="string">"system property topic ,consumer_pro_path, log_path is required !"</span>);</div><div class="line">            <span class="keyword">return</span> <span class="keyword">true</span>;</div><div class="line">        &#125;</div><div class="line">        <span class="keyword">return</span> <span class="keyword">false</span>;</div><div class="line">    &#125;</div><div class="line"></div><div class="line">    <span class="comment">/**</span></div><div class="line">     * 初始化kafka配置</div><div class="line">     * <span class="doctag">@return</span></div><div class="line">     */</div><div class="line">    <span class="function"><span class="keyword">private</span> <span class="keyword">static</span> KafkaConsumer&lt;String, String&gt; <span class="title">initKafkaConsumer</span><span class="params">()</span> </span>&#123;</div><div class="line">        KafkaConsumer&lt;String, String&gt; consumer = <span class="keyword">null</span>;</div><div class="line">        <span class="keyword">try</span> &#123;</div><div class="line">            FileInputStream inputStream = <span class="keyword">new</span> FileInputStream(<span class="keyword">new</span> File(consumerProPath)) ;</div><div class="line">            Properties properties = <span class="keyword">new</span> Properties();</div><div class="line">            properties.load(inputStream);</div><div class="line">            consumer = <span class="keyword">new</span> KafkaConsumer&lt;String, String&gt;(properties);</div><div class="line">            consumer.subscribe(Arrays.asList(topic));</div><div class="line"></div><div class="line">        &#125; <span class="keyword">catch</span> (IOException e) &#123;</div><div class="line">            LOGGER.error(<span class="string">"加载consumer.props文件出错"</span>, e);</div><div class="line">        &#125;</div><div class="line">        <span class="keyword">return</span> consumer;</div><div class="line">    &#125;</div><div class="line"></div><div class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</div><div class="line">        <span class="keyword">if</span> (initCheck())&#123;</div><div class="line">            <span class="keyword">return</span>;</div><div class="line">        &#125;</div><div class="line"></div><div class="line">        <span class="keyword">int</span> totalCount = <span class="number">0</span> ;</div><div class="line">        <span class="keyword">long</span> totalMin = <span class="number">0L</span> ;</div><div class="line">        <span class="keyword">int</span> count = <span class="number">0</span>;</div><div class="line">        KafkaConsumer&lt;String, String&gt; consumer = initKafkaConsumer();</div><div class="line"></div><div class="line">        <span class="keyword">long</span> startTime = System.currentTimeMillis() ;</div><div class="line">        <span class="comment">//消费消息</span></div><div class="line">        <span class="keyword">while</span> (<span class="keyword">true</span>) &#123;</div><div class="line">            ConsumerRecords&lt;String, String&gt; records = consumer.poll(<span class="number">200</span>);</div><div class="line">            <span class="keyword">if</span> (records.count() &lt;= <span class="number">0</span>)&#123;</div><div class="line">                <span class="keyword">continue</span> ;</div><div class="line">            &#125;</div><div class="line">            LOGGER.debug(<span class="string">"本次获取:"</span>+records.count());</div><div class="line">            count += records.count() ;</div><div class="line"></div><div class="line">            <span class="keyword">long</span> endTime = System.currentTimeMillis() ;</div><div class="line">            LOGGER.debug(<span class="string">"count="</span> +count) ;</div><div class="line">            <span class="keyword">if</span> (count &gt;= <span class="number">10000</span> )&#123;</div><div class="line">                totalCount += count ;</div><div class="line">                LOGGER.info(<span class="string">"this consumer &#123;&#125; record，use &#123;&#125; milliseconds"</span>,count,endTime-startTime);</div><div class="line">                totalMin += (endTime-startTime) ;</div><div class="line">                startTime = System.currentTimeMillis() ;</div><div class="line">                count = <span class="number">0</span> ;</div><div class="line">            &#125;</div><div class="line">            LOGGER.debug(<span class="string">"end totalCount=&#123;&#125;,min=&#123;&#125;"</span>,totalCount,totalMin);</div><div class="line"></div><div class="line">            <span class="comment">/*for (ConsumerRecord&lt;String, String&gt; record : records) &#123;</span></div><div class="line">                record.value() ;</div><div class="line">                JsonNode msg = null;</div><div class="line">                try &#123;</div><div class="line">                    msg = mapper.readTree(record.value());</div><div class="line">                &#125; catch (IOException e) &#123;</div><div class="line">                    LOGGER.error("消费消息出错", e);</div><div class="line">                &#125;</div><div class="line">                LOGGER.info("kafka receive = "+msg.toString());</div><div class="line">            &#125;*/</div><div class="line"></div><div class="line"></div><div class="line">        &#125;</div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<p>配合以下启动参数:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">-Dlog_path=/log/consumer.log -Dtopic=test -Dconsumer_pro_path=consumer.properties</div></pre></td></tr></table></figure></p>
<p>其中采用了轮询的方式获取消息，并且记录了消费过程中的数据。</p>
<p>消费者采用的配置:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div></pre></td><td class="code"><pre><div class="line">bootstrap.servers=192.168.1.2:9094</div><div class="line">group.id=group1</div><div class="line"></div><div class="line"># 自动提交</div><div class="line">enable.auto.commit=true</div><div class="line">key.deserializer=org.apache.kafka.common.serialization.StringDeserializer</div><div class="line">value.deserializer=org.apache.kafka.common.serialization.StringDeserializer</div><div class="line"></div><div class="line"># fast session timeout makes it more fun to play with failover</div><div class="line">session.timeout.ms=10000</div><div class="line"></div><div class="line"># These buffer sizes seem to be needed to avoid consumer switching to</div><div class="line"># a mode where it processes one bufferful every 5 seconds with multiple</div><div class="line"># timeouts along the way.  No idea why this happens.</div><div class="line">fetch.min.bytes=50000</div><div class="line">receive.buffer.bytes=262144</div><div class="line">max.partition.fetch.bytes=2097152</div></pre></td></tr></table></figure></p>
<p>为了简便我采用的是自动提交<code>offset</code>。</p>
<h2 id="消息存放机制"><a href="#消息存放机制" class="headerlink" title="消息存放机制"></a>消息存放机制</h2><p>谈到<code>offset</code>就必须得谈谈Kafka的消息存放机制.</p>
<p><code>Kafka</code>的消息不会因为消费了就会立即删除，所有的消息都会持久化到日志文件，并配置有过期时间，到了时间会自动删除过期数据，<em>并且不会管其中的数据是否被消费过。</em></p>
<p>由于这样的机制就必须的有一个标志来表明哪些数据已经被消费过了，<code>offset(偏移量)</code>就是这样的作用，它类似于指针指向某个数据，当消费之后<code>offset</code>就会线性的向前移动，这样一来的话消息是可以被任意消费的，只要我们修改<code>offset</code>的值即可。</p>
<p>消费过程中还有一个值得注意的是:</p>
<blockquote>
<p>同一个consumer group(group.id相等)下只能有一个消费者可以消费，这个刚开始确实会让很多人踩坑。</p>
</blockquote>
<h1 id="多线程消费"><a href="#多线程消费" class="headerlink" title="多线程消费"></a>多线程消费</h1><p>针对于单线程消费实现起来自然是比较简单，但是效率也是要大打折扣的。</p>
<p>为此我做了一个测试，使用之前的单线程消费120009条数据的结果如下:</p>
<p><img src="https://i.loli.net/2019/05/08/5cd1ba8705aac.jpg" alt=""><br>总共花了12450毫秒。</p>
<p>那么换成多线程消费怎么实现呢？</p>
<p>我们可以利用<code>partition</code>的分区特性来提高消费能力，单线程的时候等于是一个线程要把所有分区里的数据都消费一遍，如果换成多线程就可以让一个线程只消费一个分区,这样效率自然就提高了，所以线程数<code>coreSize&lt;=partition</code>。</p>
<p>首先来看下入口:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div></pre></td><td class="code"><pre><div class="line">public class ConsumerThreadMain &#123;</div><div class="line">    private static String brokerList = &quot;localhost:9094&quot;;</div><div class="line">    private static String groupId = &quot;group1&quot;;</div><div class="line">    private static String topic = &quot;test&quot;;</div><div class="line"></div><div class="line">    /**</div><div class="line">     * 线程数量</div><div class="line">     */</div><div class="line">    private static int threadNum = 3;</div><div class="line"></div><div class="line">    public static void main(String[] args) &#123;</div><div class="line"></div><div class="line"></div><div class="line">        ConsumerGroup consumerGroup = new ConsumerGroup(threadNum, groupId, topic, brokerList);</div><div class="line">        consumerGroup.execute();</div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<p>其中的<code>ConsumerGroup</code>类:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div></pre></td><td class="code"><pre><div class="line">public class ConsumerGroup &#123;</div><div class="line">    private static Logger LOGGER = LoggerFactory.getLogger(ConsumerGroup.class);</div><div class="line">    /**</div><div class="line">     * 线程池</div><div class="line">     */</div><div class="line">    private ExecutorService threadPool;</div><div class="line"></div><div class="line">    private List&lt;ConsumerCallable&gt; consumers ;</div><div class="line"></div><div class="line">    public ConsumerGroup(int threadNum, String groupId, String topic, String brokerList) &#123;</div><div class="line">        ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()</div><div class="line">                .setNameFormat(&quot;consumer-pool-%d&quot;).build();</div><div class="line"></div><div class="line">        threadPool = new ThreadPoolExecutor(threadNum, threadNum,</div><div class="line">                0L, TimeUnit.MILLISECONDS,</div><div class="line">                new LinkedBlockingQueue&lt;Runnable&gt;(1024), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());</div><div class="line"></div><div class="line"></div><div class="line">        consumers = new ArrayList&lt;ConsumerCallable&gt;(threadNum);</div><div class="line">        for (int i = 0; i &lt; threadNum; i++) &#123;</div><div class="line">            ConsumerCallable consumerThread = new ConsumerCallable(brokerList, groupId, topic);</div><div class="line">            consumers.add(consumerThread);</div><div class="line">        &#125;</div><div class="line">    &#125;</div><div class="line"></div><div class="line">    /**</div><div class="line">     * 执行任务</div><div class="line">     */</div><div class="line">    public void execute() &#123;</div><div class="line">        long startTime = System.currentTimeMillis() ;</div><div class="line">        for (ConsumerCallable runnable : consumers) &#123;</div><div class="line">            Future&lt;ConsumerFuture&gt; future = threadPool.submit(runnable) ;</div><div class="line">        &#125;</div><div class="line">        if (threadPool.isShutdown())&#123;</div><div class="line">            long endTime = System.currentTimeMillis() ;</div><div class="line">            LOGGER.info(&quot;main thread use &#123;&#125; Millis&quot; ,endTime -startTime) ;</div><div class="line">        &#125;</div><div class="line">        threadPool.shutdown();</div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure></p>
<p>最后真正的执行逻辑<code>ConsumerCallable</code>:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div><div class="line">45</div><div class="line">46</div><div class="line">47</div><div class="line">48</div><div class="line">49</div><div class="line">50</div><div class="line">51</div><div class="line">52</div><div class="line">53</div><div class="line">54</div><div class="line">55</div><div class="line">56</div><div class="line">57</div><div class="line">58</div><div class="line">59</div><div class="line">60</div><div class="line">61</div><div class="line">62</div><div class="line">63</div><div class="line">64</div><div class="line">65</div><div class="line">66</div><div class="line">67</div><div class="line">68</div><div class="line">69</div><div class="line">70</div><div class="line">71</div><div class="line">72</div><div class="line">73</div><div class="line">74</div><div class="line">75</div><div class="line">76</div><div class="line">77</div><div class="line">78</div><div class="line">79</div><div class="line">80</div></pre></td><td class="code"><pre><div class="line">public class ConsumerCallable implements Callable&lt;ConsumerFuture&gt; &#123;</div><div class="line">    private static Logger LOGGER = LoggerFactory.getLogger(ConsumerCallable.class);</div><div class="line"></div><div class="line">    private AtomicInteger totalCount = new AtomicInteger() ;</div><div class="line">    private AtomicLong totalTime = new AtomicLong() ;</div><div class="line"></div><div class="line">    private AtomicInteger count = new AtomicInteger() ;</div><div class="line"></div><div class="line">    /**</div><div class="line">     * 每个线程维护KafkaConsumer实例</div><div class="line">     */</div><div class="line">    private final KafkaConsumer&lt;String, String&gt; consumer;</div><div class="line"></div><div class="line">    public ConsumerCallable(String brokerList, String groupId, String topic) &#123;</div><div class="line">        Properties props = new Properties();</div><div class="line">        props.put(&quot;bootstrap.servers&quot;, brokerList);</div><div class="line">        props.put(&quot;group.id&quot;, groupId);</div><div class="line">        //自动提交位移</div><div class="line">        props.put(&quot;enable.auto.commit&quot;, &quot;true&quot;);</div><div class="line">        props.put(&quot;auto.commit.interval.ms&quot;, &quot;1000&quot;);</div><div class="line">        props.put(&quot;session.timeout.ms&quot;, &quot;30000&quot;);</div><div class="line">        props.put(&quot;key.deserializer&quot;, &quot;org.apache.kafka.common.serialization.StringDeserializer&quot;);</div><div class="line">        props.put(&quot;value.deserializer&quot;, &quot;org.apache.kafka.common.serialization.StringDeserializer&quot;);</div><div class="line">        this.consumer = new KafkaConsumer&lt;&gt;(props);</div><div class="line">        consumer.subscribe(Arrays.asList(topic));</div><div class="line">    &#125;</div><div class="line"></div><div class="line"></div><div class="line">    /**</div><div class="line">     * Computes a result, or throws an exception if unable to do so.</div><div class="line">     *</div><div class="line">     * @return computed result</div><div class="line">     * @throws Exception if unable to compute a result</div><div class="line">     */</div><div class="line">    @Override</div><div class="line">    public ConsumerFuture call() throws Exception &#123;</div><div class="line">        boolean flag = true;</div><div class="line">        int failPollTimes = 0 ;</div><div class="line">        long startTime = System.currentTimeMillis() ;</div><div class="line">        while (flag) &#123;</div><div class="line">            // 使用200ms作为获取超时时间</div><div class="line">            ConsumerRecords&lt;String, String&gt; records = consumer.poll(200);</div><div class="line">            if (records.count() &lt;= 0)&#123;</div><div class="line">                failPollTimes ++ ;</div><div class="line"></div><div class="line">                if (failPollTimes &gt;= 20)&#123;</div><div class="line">                    LOGGER.debug(&quot;达到&#123;&#125;次数，退出 &quot;,failPollTimes);</div><div class="line">                    flag = false ;</div><div class="line">                &#125;</div><div class="line"></div><div class="line">                continue ;</div><div class="line">            &#125;</div><div class="line"></div><div class="line">            //获取到之后则清零</div><div class="line">            failPollTimes = 0 ;</div><div class="line"></div><div class="line">            LOGGER.debug(&quot;本次获取:&quot;+records.count());</div><div class="line">            count.addAndGet(records.count()) ;</div><div class="line">            totalCount.addAndGet(count.get()) ;</div><div class="line">            long endTime = System.currentTimeMillis() ;</div><div class="line">            if (count.get() &gt;= 10000 )&#123;</div><div class="line">                LOGGER.info(&quot;this consumer &#123;&#125; record，use &#123;&#125; milliseconds&quot;,count,endTime-startTime);</div><div class="line">                totalTime.addAndGet(endTime-startTime) ;</div><div class="line">                startTime = System.currentTimeMillis() ;</div><div class="line">                count = new AtomicInteger();</div><div class="line">            &#125;</div><div class="line">            LOGGER.debug(&quot;end totalCount=&#123;&#125;,min=&#123;&#125;&quot;,totalCount,totalTime);</div><div class="line"></div><div class="line">            /*for (ConsumerRecord&lt;String, String&gt; record : records) &#123;</div><div class="line">                // 简单地打印消息</div><div class="line">                LOGGER.debug(record.value() + &quot; consumed &quot; + record.partition() +</div><div class="line">                        &quot; message with offset: &quot; + record.offset());</div><div class="line">            &#125;*/</div><div class="line">        &#125;</div><div class="line"></div><div class="line">        ConsumerFuture consumerFuture = new ConsumerFuture(totalCount.get(),totalTime.get()) ;</div><div class="line">        return consumerFuture ;</div><div class="line"></div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure></p>
<p>理一下逻辑:</p>
<blockquote>
<p>其实就是初始化出三个消费者实例，用于三个线程消费。其中加入了一些统计，最后也是消费120009条数据结果如下。</p>
</blockquote>
<p><img src="https://i.loli.net/2019/05/08/5cd1ba8b41500.jpg" alt=""></p>
<p>由于是并行运行，可见消费120009条数据可以提高2秒左右，当数据以更高的数量级提升后效果会更加明显。</p>
<p>但这也有一些弊端:</p>
<ul>
<li>灵活度不高，当分区数量变更之后不能自适应调整。</li>
<li>消费逻辑和处理逻辑在同一个线程，如果处理逻辑较为复杂会影响效率，耦合也较高。当然这个处理逻辑可以再通过一个内部队列发出去由另外的程序来处理也是可以的。</li>
</ul>
<h1 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h1><p><code>Kafka</code>的知识点还是较多，<code>Kafka</code>的使用也远不这些。之后会继续分享一些关于<code>Kafka</code>监控等相关内容。</p>
<blockquote>
<p>项目地址：<a href="https://github.com/crossoverJie/SSM.git" target="_blank" rel="external">https://github.com/crossoverJie/SSM.git</a></p>
<p>个人博客：<a href="http://crossoverjie.top">http://crossoverjie.top</a>。</p>
</blockquote>

      
    </div>

    <div>
      
        
<div id="wechat_subscriber" style="display: block； padding: 10px 0; margin: 20px auto; width: 100%; text-align: center">
    <img id="wechat_subscriber_qcode" src="/uploads/weixinfooter1.jpg" alt="crossoverJie wechat" style="width: 200px; max-width: 100%;"/>
    <div>我很有眼光！</div>
</div>

      
    </div>

    <div>
      
        
  <div style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center;">
    <div>请我吃🍗</div>
    <button id="rewardButton" disable="enable" onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}">
      <span>赏</span>
    </button>
    <div id="QR" style="display: none;">
      
        <div id="wechat" style="display: inline-block">
          <img id="wechat_qr" src="/weixin-reward-image.jpg" alt="crossoverJie WeChat Pay"/>
          <p>微信打赏</p>
        </div>
      
      
        <div id="alipay" style="display: inline-block">
          <img id="alipay_qr" src="/alipay-reward-image.jpg" alt="crossoverJie Alipay"/>
          <p>支付宝打赏</p>
        </div>
      
    </div>
  </div>


      
    </div>

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/Java/" rel="tag">#Java</a>
          
            <a href="/tags/Kafka/" rel="tag">#Kafka</a>
          
        </div>
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2017/09/20/sbc5/" rel="next" title="sbc(五)Hystrix-服务容错与保护">
                <i class="fa fa-chevron-left"></i> sbc(五)Hystrix-服务容错与保护
              </a>
            
          </div>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2017/11/05/translation1-12 cool things you can do with GitHub/" rel="prev" title="【译】你可以用GitHub做的12件 Cool 事情">
                【译】你可以用GitHub做的12件 Cool 事情 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          

  <p>热评文章</p>
  <div class="ds-top-threads" data-range="weekly" data-num-items="4"></div>


          
  <div class="comments" id="comments">
    
      <div id="disqus_thread">
        <noscript>
          Please enable JavaScript to view the
          <a href="//disqus.com/?ref_noscript">comments powered by Disqus.</a>
        </noscript>
      </div>
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            Table of Contents
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            Overview
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel ">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/uploads/crossoverjie.jpg"
               alt="crossoverJie" />
          <p class="site-author-name" itemprop="name">crossoverJie</p>
          <p class="site-description motion-element" itemprop="description">You never know what you can do till you try.</p>
        </div>
        <nav class="site-state motion-element">
          <div class="site-state-item site-state-posts">
            <a href="/archives">
              <span class="site-state-item-count">116</span>
              <span class="site-state-item-name">posts</span>
            </a>
          </div>

          
            <div class="site-state-item site-state-categories">
              <a href="/categories">
                <span class="site-state-item-count">45</span>
                <span class="site-state-item-name">categories</span>
              </a>
            </div>
          

          
            <div class="site-state-item site-state-tags">
              <a href="/tags">
                <span class="site-state-item-count">109</span>
                <span class="site-state-item-name">tags</span>
              </a>
            </div>
          

        </nav>

        
          <div class="feed-link motion-element">
            <a href="/atom.xml" rel="alternate">
              <i class="fa fa-rss"></i>
              RSS
            </a>
          </div>
        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/crossoverJie" target="_blank" title="GitHub">
                  
                    <i class="fa fa-fw fa-github"></i>
                  
                  GitHub
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://www.jianshu.com/users/e2d07947c112/latest_articles" target="_blank" title="简书">
                  
                    <i class="fa fa-fw fa-book"></i>
                  
                  简书
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="https://juejin.im/user/576d4aaf7db2a20054ea4544" target="_blank" title="掘金">
                  
                    <i class="fa fa-fw fa-bookmark"></i>
                  
                  掘金
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="https://twitter.com/crossoverJie" target="_blank" title="Twitter">
                  
                    <i class="fa fa-fw fa-twitter"></i>
                  
                  Twitter
                </a>
              </span>
            
          
        </div>

        
        

        
        
          <div class="links-of-blogroll motion-element links-of-blogroll-inline">
            <div class="links-of-blogroll-title">
              <i class="fa  fa-fw fa-globe"></i>
              友情链接
            </div>
            <ul class="links-of-blogroll-list">
              
                <li class="links-of-blogroll-item">
                  <a href="http://wuchong.me" title="Jark's Blog" target="_blank">Jark's Blog</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://tengj.top" title="嘟嘟独立博客" target="_blank">嘟嘟独立博客</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://yemengying.com/" title="Giraffe Home" target="_blank">Giraffe Home</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="https://blog.jamespan.me/" title="潘小鶸(ruò)" target="_blank">潘小鶸(ruò)</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://fangjian0423.github.io/" title="Format's Notes" target="_blank">Format's Notes</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="https://yuzhouwan.com/" title="Benedict Jin" target="_blank">Benedict Jin</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://blog.didispace.com/" title="程序猿DD" target="_blank">程序猿DD</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="https://blog.52itstyle.vip/" title="小柒博客" target="_blank">小柒博客</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://cmsblogs.com/" title="Java技术驿站" target="_blank">Java技术驿站</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="https://vim.ink/" title="vim 教程网" target="_blank">vim 教程网</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="https://www.jitwxs.cn" title="jitwxs" target="_blank">jitwxs</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://www.javaboy.org" title="江南一点雨" target="_blank">江南一点雨</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://www.liangsonghua.me" title="松花皮蛋的黑板报" target="_blank">松花皮蛋的黑板报</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="https://www.fi-ads.com" title="Future iDeal" target="_blank">Future iDeal</a>
                </li>
              
            </ul>
          </div>
        

      </section>

      
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">
            
              
            
            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#前言"><span class="nav-number">1.</span> <span class="nav-text">前言</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#何时使用MQ"><span class="nav-number">2.</span> <span class="nav-text">何时使用MQ</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Kafka简介"><span class="nav-number">3.</span> <span class="nav-text">Kafka简介</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#创建Topic"><span class="nav-number">4.</span> <span class="nav-text">创建Topic</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Kafka生产者"><span class="nav-number">5.</span> <span class="nav-text">Kafka生产者</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Kafka消费者"><span class="nav-number">6.</span> <span class="nav-text">Kafka消费者</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#消息存放机制"><span class="nav-number">6.1.</span> <span class="nav-text">消息存放机制</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#多线程消费"><span class="nav-number">7.</span> <span class="nav-text">多线程消费</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#总结"><span class="nav-number">8.</span> <span class="nav-text">总结</span></a></li></ol></div>
            
          </div>
        </section>
      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy;  2016 - 
  <span itemprop="copyrightYear">2019</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">crossoverJie</span>
</div>

<div class="powered-by">
  Powered by <a class="theme-link" href="http://hexo.io">Hexo</a>
</div>

<div class="theme-info">
  Theme -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Pisces
  </a>
</div>

        

<div class="busuanzi-count">

  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv"><i class="fa fa-user"></i><span class="busuanzi-value" id="busuanzi_value_site_uv"></span>人数</span>
  

  
    <span class="site-pv">您是第<span class="busuanzi-value" id="busuanzi_value_site_pv"></span>位童鞋</span>
  
  
</div>



        
      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  



  
  <script type="text/javascript" src="/vendors/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/vendors/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/vendors/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/vendors/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/vendors/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/vendors/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.0.1"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.0.1"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.0.1"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.0.1"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.0.1"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.0.1"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.0.1"></script>



  



  

    <script type="text/javascript">
      var disqus_shortname = 'crossoverjie';
      var disqus_identifier = '2017/10/20/SSM17/';
      var disqus_title = "SSM(十七) MQ应用";
      var disqus_url = 'http://crossoverjie.top/2017/10/20/SSM17/';

      function run_disqus_script(disqus_script){
        var dsq = document.createElement('script');
        dsq.type = 'text/javascript';
        dsq.async = true;
        dsq.src = '//' + disqus_shortname + '.disqus.com/' + disqus_script;
        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
      }

      run_disqus_script('count.js');
      
        run_disqus_script('embed.js');
      
    </script>
  




  
  
  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length == 0) {
       search_path = "search.xml";
    }
    var path = "/" + search_path;
    // monitor main search box;

    function proceedsearch() {
      $("body").append('<div class="popoverlay">').css('overflow', 'hidden');
      $('.popup').toggle();

    }
    // search function;
    var searchFunc = function(path, search_id, content_id) {
    'use strict';
    $.ajax({
        url: path,
        dataType: "xml",
        async: true,
        success: function( xmlResponse ) {
            // get the contents from search data
            isfetched = true;
            $('.popup').detach().appendTo('.header-inner');
            var datas = $( "entry", xmlResponse ).map(function() {
                return {
                    title: $( "title", this ).text(),
                    content: $("content",this).text(),
                    url: $( "url" , this).text()
                };
            }).get();
            var $input = document.getElementById(search_id);
            var $resultContent = document.getElementById(content_id);
            $input.addEventListener('input', function(){
                var matchcounts = 0;
                var str='<ul class=\"search-result-list\">';                
                var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
                $resultContent.innerHTML = "";
                if (this.value.trim().length > 1) {
                // perform local searching
                datas.forEach(function(data) {
                    var isMatch = true;
                    var content_index = [];
                    var data_title = data.title.trim().toLowerCase();
                    var data_content = data.content.trim().replace(/<[^>]+>/g,"").toLowerCase();
                    var data_url = data.url;
                    var index_title = -1;
                    var index_content = -1;
                    var first_occur = -1;
                    // only match artiles with not empty titles and contents
                    if(data_title != '' && data_content != '') {
                        keywords.forEach(function(keyword, i) {
                            index_title = data_title.indexOf(keyword);
                            index_content = data_content.indexOf(keyword);
                            if( index_title < 0 && index_content < 0 ){
                                isMatch = false;
                            } else {
                                if (index_content < 0) {
                                    index_content = 0;
                                }
                                if (i == 0) {
                                    first_occur = index_content;
                                }
                            }
                        });
                    }
                    // show search results
                    if (isMatch) {
                        matchcounts += 1;
                        str += "<li><a href='"+ data_url +"' class='search-result-title'>"+ data_title +"</a>";
                        var content = data.content.trim().replace(/<[^>]+>/g,"");
                        if (first_occur >= 0) {
                            // cut out 100 characters
                            var start = first_occur - 20;
                            var end = first_occur + 80;
                            if(start < 0){
                                start = 0;
                            }
                            if(start == 0){
                                end = 50;
                            }
                            if(end > content.length){
                                end = content.length;
                            }
                            var match_content = content.substring(start, end);
                            // highlight all keywords
                            keywords.forEach(function(keyword){
                                var regS = new RegExp(keyword, "gi");
                                match_content = match_content.replace(regS, "<b class=\"search-keyword\">"+keyword+"</b>");
                            });
                            
                            str += "<p class=\"search-result\">" + match_content +"...</p>"
                        }
                        str += "</li>";
                    }
                })};
                str += "</ul>";
                if (matchcounts == 0) { str = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>' }
                if (keywords == "") { str = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>' }
                $resultContent.innerHTML = str;
            });
            proceedsearch();
        }
    });}

    // handle and trigger popup window;
    $('.popup-trigger').mousedown(function(e) {
      e.stopPropagation();
      if (isfetched == false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };

    });

    $('.popup-btn-close').click(function(e){
      $('.popup').hide();
      $(".popoverlay").remove();
      $('body').css('overflow', '');
    });
    $('.popup').click(function(e){
      e.stopPropagation();
    });
  </script>

  

  

  
  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.1.js"></script>
  <script>AV.initialize("Qv6ckEtL1pe3PJD10qoOLKtg-gzGzoHsz", "NXiHFodQfmI8oxkK6IThhjrF");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for(var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if( countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(counter.get('time'));
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            /* Set ACL */
            var acl = new AV.ACL();
            acl.setPublicReadAccess(true);
            acl.setPublicWriteAccess(true);
            newcounter.setACL(acl);
            /* End Set ACL */
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script>



  
<script type="text/javascript" src="/js/src/particle.js" count="50" zindex="-2" opacity="1" color="0,104,183"></script>
</body>
</html>
